{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################\n",
    "### 对于连续性变量，直接分箱\n",
    "### 定义一些分箱用到的辅助函数\n",
    "##############################################\n",
    "\n",
    "#当连续变量的初始取值集合太多时（>100），我们先对其进行初步划分\n",
    "def SplitData(df, col, numOfSplit, special_attribute=[]):\n",
    "    '''\n",
    "    :param df: 按照col排序后的数据集\n",
    "    :param col: 待分箱的变量\n",
    "    :param numOfSplit: 切分的组别数\n",
    "    :param special_attribute: 在切分数据集的时候，某些特殊值需要排除在外\n",
    "    :return: 在原数据集上增加一列，把原始细粒度的col重新划分成粗粒度的值，便于分箱中的合并处理\n",
    "    '''\n",
    "    df2 = df.copy()\n",
    "    if special_attribute != []:\n",
    "        df2 = df.loc[~df[col].isin(special_attribute)]\n",
    "    N = df2.shape[0]\n",
    "    n = N//numOfSplit #每组样本数\n",
    "    splitPointIndex = [i*n for i in range(1,numOfSplit)] #分割点的下标\n",
    "    rawValues = sorted(list(df2[col])) #对取值进行排序\n",
    "    splitPoint = [rawValues[i] for i in splitPointIndex] #分割点的取值\n",
    "    splitPoint = sorted(list(set(splitPoint)))\n",
    "    return splitPoint\n",
    "\n",
    "#根据分割点将x映射至相应的组中\n",
    "def AssignGroup(x, bin):\n",
    "    '''\n",
    "    :param x: 某个变量的某个取值\n",
    "    :param bin: 上述变量的分箱结果\n",
    "    :return: x在分箱结果下的映射\n",
    "    '''\n",
    "    N = len(bin)\n",
    "    if x<=min(bin):\n",
    "        return min(bin)\n",
    "    elif x>max(bin):\n",
    "        return 10e10\n",
    "    else:\n",
    "        for i in range(N-1):\n",
    "            if bin[i] < x <= bin[i+1]:\n",
    "                return bin[i+1]\n",
    "\n",
    "def AssignBin(x, cutOffPoints,special_attribute=[]):\n",
    "    '''\n",
    "    :param x: 某个变量的某个取值\n",
    "    :param cutOffPoints: 上述变量的分箱结果，用切分点表示\n",
    "    :param special_attribute:  不参与分箱的特殊取值\n",
    "    :return: 分箱后的对应的第几个箱，从0开始\n",
    "    for example, if cutOffPoints = [10,20,30], if x = 7, return Bin 0. If x = 35, return Bin 3\n",
    "    '''\n",
    "    numBin = len(cutOffPoints) + 1 + len(special_attribute)\n",
    "    if x in special_attribute:\n",
    "        i = special_attribute.index(x)+1\n",
    "        return 'Bin {}'.format(0-i)\n",
    "    if x<=cutOffPoints[0]:\n",
    "        return 'Bin 0'\n",
    "    elif x > cutOffPoints[-1]:\n",
    "        return 'Bin {}'.format(numBin-1)\n",
    "    else:\n",
    "        for i in range(0,numBin-1):\n",
    "            if cutOffPoints[i] < x <=  cutOffPoints[i+1]:\n",
    "                return 'Bin {}'.format(i+1)\n",
    "\n",
    "## 判断某变量的坏样本率是否单调，如果不单调，需缩减分箱个数\n",
    "def BadRateMonotone(df, sortByVar, target,special_attribute = []):\n",
    "    '''\n",
    "    :param df: 包含检验坏样本率的变量，和目标变量\n",
    "    :param sortByVar: 需要检验坏样本率的变量\n",
    "    :param target: 目标变量，0、1表示好、坏\n",
    "    :param special_attribute: 不参与检验的特殊值\n",
    "    :return: 坏样本率单调与否\n",
    "    '''\n",
    "    df2 = df.loc[~df[sortByVar].isin(special_attribute)]\n",
    "    if len(set(df2[sortByVar])) <= 2:\n",
    "        return True\n",
    "    regroup = BinBadRate(df2, sortByVar, target)[1]\n",
    "    combined = zip(regroup['total'],regroup['bad'])\n",
    "    badRate = [x[1]*1.0/x[0] for x in combined]\n",
    "    badRateNotMonotone = [badRate[i]<badRate[i+1] and badRate[i] < badRate[i-1] or badRate[i]>badRate[i+1] and badRate[i] > badRate[i-1]\n",
    "                       for i in range(1,len(badRate)-1)]\n",
    "    if True in badRateNotMonotone:\n",
    "        return False\n",
    "    else:\n",
    "        return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################\n",
    "### 计算卡方统计量\n",
    "##############################################\n",
    "\n",
    "def Chi2(df, total_col, bad_col, overallRate):\n",
    "    '''\n",
    "    :param df: 包含全部样本总计与坏样本总计的数据框\n",
    "    :param total_col: 全部样本的个数\n",
    "    :param bad_col: 坏样本的个数\n",
    "    :param overallRate: 全体样本的坏样本占比\n",
    "    :return: 卡方值\n",
    "    '''\n",
    "    df2 = df.copy()\n",
    "    # 期望坏样本个数＝全部样本个数*平均坏样本占比\n",
    "    df2['good']=df2['total']-df2['bad']\n",
    "    df2['expected_bad'] = df[total_col].apply(lambda x: x*overallRate)\n",
    "    df2['expected_good']=df[total_col].apply(lambda x:x*(1-overallRate))\n",
    "    combined_bad = zip(df2['expected_bad'], df2[bad_col])\n",
    "    combined_good=zip(df2['expected_good'],df2['good'])\n",
    "    chi_bad = [(i[0]-i[1])**2/i[0] for i in combined_bad]\n",
    "    chi_good=[(i[0]-i[1])**2/i[0] for i in combined_good]\n",
    "    df=(df2.shape[0]-1)*(2-1) #自由度\n",
    "    chi2 = (sum(chi_bad)+sum(chi_good))/df\n",
    "    return chi2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################\n",
    "### 卡方分箱\n",
    "##############################################\n",
    "\n",
    "### ChiMerge_MaxInterval: split the continuous variable using Chi-square value by specifying the max number of intervals\n",
    "def ChiMerge(df, col, target, max_interval=5,special_attribute=[],minBinPcnt=0):\n",
    "    '''\n",
    "    :param df: 包含目标变量与分箱属性的数据框\n",
    "    :param col: 需要分箱的属性\n",
    "    :param target: 目标变量，取值0或1\n",
    "    :param max_interval: 最大分箱数。如果原始属性的取值个数低于该参数，不执行这段函数\n",
    "    :param special_attribute: 不参与分箱的属性取值\n",
    "    :param minBinPcnt：最小箱的占比，默认为0\n",
    "    :return: 分箱结果\n",
    "    '''\n",
    "    colLevels = sorted(list(set(df[col])))\n",
    "    N_distinct = len(colLevels)#不同的取值个数\n",
    "    if N_distinct <= max_interval:  #如果原始属性的取值个数低于max_interval，不执行这段函数\n",
    "        print (\"The number of original levels for {} is less than or equal to max intervals\".format(col))\n",
    "        return colLevels[:-1]\n",
    "    else:\n",
    "        if len(special_attribute)>=1:\n",
    "            df1 = df.loc[df[col].isin(special_attribute)]\n",
    "            df2 = df.loc[~df[col].isin(special_attribute)]\n",
    "        else:\n",
    "            df2 = df.copy()\n",
    "        N_distinct = len(list(set(df2[col])))#该特征不同的取值\n",
    "\n",
    "        # 步骤一: 通过col对数据集进行分组，求出每组的总样本数与坏样本数\n",
    "        if N_distinct > 100:\n",
    "            split_x = SplitData(df2, col, 100)\n",
    "            df2['temp'] = df2[col].map(lambda x: AssignGroup(x, split_x))\n",
    "        else:\n",
    "            df2['temp'] = df2[col]\n",
    "        # 总体bad rate将被用来计算expected bad count\n",
    "        (binBadRate, regroup, overallRate) = BinBadRate(df2, 'temp', target, grantRateIndicator=1)\n",
    "\n",
    "        # 首先，每个单独的属性值将被分为单独的一组\n",
    "        # 对属性值进行排序，然后两两组别进行合并\n",
    "        colLevels = sorted(list(set(df2['temp'])))\n",
    "        groupIntervals = [[i] for i in colLevels]\n",
    "\n",
    "        # 步骤二：建立循环，不断合并最优的相邻两个组别，直到：\n",
    "        # 1，最终分裂出来的分箱数<＝预设的最大分箱数\n",
    "        # 2，每箱的占比不低于预设值（可选）\n",
    "        # 3，每箱同时包含好坏样本\n",
    "        # 如果有特殊属性，那么最终分裂出来的分箱数＝预设的最大分箱数－特殊属性的个数\n",
    "        split_intervals = max_interval - len(special_attribute)\n",
    "        while (len(groupIntervals) > split_intervals):  # 终止条件: 当前分箱数＝预设的分箱数\n",
    "            # 每次循环时, 计算合并相邻组别后的卡方值。具有最小卡方值的合并方案，是最优方案\n",
    "            chisqList = []\n",
    "            for k in range(len(groupIntervals)-1):\n",
    "                temp_group = groupIntervals[k] + groupIntervals[k+1]\n",
    "                df2b = regroup.loc[regroup['temp'].isin(temp_group)]\n",
    "                chisq = Chi2(df2b, 'total', 'bad', overallRate)\n",
    "                chisqList.append(chisq)\n",
    "            best_comnbined = chisqList.index(min(chisqList))\n",
    "            groupIntervals[best_comnbined] = groupIntervals[best_comnbined] + groupIntervals[best_comnbined+1]\n",
    "            # after combining two intervals, we need to remove one of them\n",
    "            groupIntervals.remove(groupIntervals[best_comnbined+1])\n",
    "        groupIntervals = [sorted(i) for i in groupIntervals]\n",
    "        cutOffPoints = [max(i) for i in groupIntervals[:-1]]\n",
    "\n",
    "        # 检查是否有箱没有好或者坏样本。如果有，需要跟相邻的箱进行合并，直到每箱同时包含好坏样本\n",
    "        groupedvalues = df2['temp'].apply(lambda x: AssignBin(x, cutOffPoints))\n",
    "        df2['temp_Bin'] = groupedvalues\n",
    "        (binBadRate,regroup) = BinBadRate(df2, 'temp_Bin', target)\n",
    "        [minBadRate, maxBadRate] = [min(binBadRate.values()),max(binBadRate.values())]\n",
    "        while minBadRate ==0 or maxBadRate == 1:\n",
    "            # 找出全部为好／坏样本的箱\n",
    "            indexForBad01 = regroup[regroup['bad_rate'].isin([0,1])].temp_Bin.tolist()\n",
    "            bin=indexForBad01[0]\n",
    "            # 如果是最后一箱，则需要和上一个箱进行合并，也就意味着分裂点cutOffPoints中的最后一个需要移除\n",
    "            if bin == max(regroup.temp_Bin):\n",
    "                cutOffPoints = cutOffPoints[:-1]\n",
    "            # 如果是第一箱，则需要和下一个箱进行合并，也就意味着分裂点cutOffPoints中的第一个需要移除\n",
    "            elif bin == min(regroup.temp_Bin):\n",
    "                cutOffPoints = cutOffPoints[1:]\n",
    "            # 如果是中间的某一箱，则需要和前后中的一个箱进行合并，依据是较小的卡方值\n",
    "            else:\n",
    "                # 和前一箱进行合并，并且计算卡方值\n",
    "                currentIndex = list(regroup.temp_Bin).index(bin)\n",
    "                prevIndex = list(regroup.temp_Bin)[currentIndex - 1]\n",
    "                df3 = df2.loc[df2['temp_Bin'].isin([prevIndex, bin])]\n",
    "                (binBadRate, df2b) = BinBadRate(df3, 'temp_Bin', target)\n",
    "                chisq1 = Chi2(df2b, 'total', 'bad', overallRate)\n",
    "                # 和后一箱进行合并，并且计算卡方值\n",
    "                laterIndex = list(regroup.temp_Bin)[currentIndex + 1]\n",
    "                df3b = df2.loc[df2['temp_Bin'].isin([laterIndex, bin])]\n",
    "                (binBadRate, df2b) = BinBadRate(df3b, 'temp_Bin', target)\n",
    "                chisq2 = Chi2(df2b, 'total', 'bad', overallRate)\n",
    "                if chisq1 < chisq2:\n",
    "                    cutOffPoints.remove(cutOffPoints[currentIndex - 1])\n",
    "                else:\n",
    "                    cutOffPoints.remove(cutOffPoints[currentIndex])\n",
    "            # 完成合并之后，需要再次计算新的分箱准则下，每箱是否同时包含好坏样本\n",
    "            groupedvalues = df2['temp'].apply(lambda x: AssignBin(x, cutOffPoints))\n",
    "            df2['temp_Bin'] = groupedvalues\n",
    "            (binBadRate, regroup) = BinBadRate(df2, 'temp_Bin', target)\n",
    "            [minBadRate, maxBadRate] = [min(binBadRate.values()), max(binBadRate.values())]\n",
    "        # 需要检查分箱后的最小占比\n",
    "        if minBinPcnt > 0:\n",
    "            groupedvalues = df2['temp'].apply(lambda x: AssignBin(x, cutOffPoints))\n",
    "            df2['temp_Bin'] = groupedvalues\n",
    "            valueCounts = groupedvalues.value_counts().to_frame()\n",
    "            valueCounts['pcnt'] = valueCounts['temp'].apply(lambda x: x * 1.0 / N)\n",
    "            valueCounts = valueCounts.sort_index()\n",
    "            minPcnt = min(valueCounts['pcnt'])\n",
    "            while minPcnt < minBinPcnt and len(cutOffPoints) > 2:\n",
    "                # 找出占比最小的箱\n",
    "                indexForMinPcnt = valueCounts[valueCounts['pcnt'] == minPcnt].index.tolist()[0]\n",
    "                # 如果占比最小的箱是最后一箱，则需要和上一个箱进行合并，也就意味着分裂点cutOffPoints中的最后一个需要移除\n",
    "                if indexForMinPcnt == max(valueCounts.index):\n",
    "                    cutOffPoints = cutOffPoints[:-1]\n",
    "                # 如果占比最小的箱是第一箱，则需要和下一个箱进行合并，也就意味着分裂点cutOffPoints中的第一个需要移除\n",
    "                elif indexForMinPcnt == min(valueCounts.index):\n",
    "                    cutOffPoints = cutOffPoints[1:]\n",
    "                # 如果占比最小的箱是中间的某一箱，则需要和前后中的一个箱进行合并，依据是较小的卡方值\n",
    "                else:\n",
    "                    # 和前一箱进行合并，并且计算卡方值\n",
    "                    currentIndex = list(valueCounts.index).index(indexForMinPcnt)\n",
    "                    prevIndex = list(valueCounts.index)[currentIndex - 1]\n",
    "                    df3 = df2.loc[df2['temp_Bin'].isin([prevIndex, indexForMinPcnt])]\n",
    "                    (binBadRate, df2b) = BinBadRate(df3, 'temp_Bin', target)\n",
    "                    chisq1 = Chi2(df2b, 'total', 'bad', overallRate)\n",
    "                    # 和后一箱进行合并，并且计算卡方值\n",
    "                    laterIndex = list(valueCounts.index)[currentIndex + 1]\n",
    "                    df3b = df2.loc[df2['temp_Bin'].isin([laterIndex, indexForMinPcnt])]\n",
    "                    (binBadRate, df2b) = BinBadRate(df3b, 'temp_Bin', target)\n",
    "                    chisq2 = Chi2(df2b, 'total', 'bad', overallRate)\n",
    "                    if chisq1 < chisq2:\n",
    "                        cutOffPoints.remove(cutOffPoints[currentIndex - 1])\n",
    "                    else:\n",
    "                        cutOffPoints.remove(cutOffPoints[currentIndex])\n",
    "        cutOffPoints = special_attribute + cutOffPoints\n",
    "        return cutOffPoints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>V1</th>\n",
       "      <th>V2</th>\n",
       "      <th>V3</th>\n",
       "      <th>V4</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.7</td>\n",
       "      <td>3.2</td>\n",
       "      <td>1.3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.6</td>\n",
       "      <td>3.1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.6</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    V1   V2   V3   V4  y\n",
       "0  5.1  3.5  1.4  0.2  0\n",
       "1  4.9  3.0  1.4  0.2  0\n",
       "2  4.7  3.2  1.3  0.2  0\n",
       "3  4.6  3.1  1.5  0.2  0\n",
       "4  5.0  3.6  1.4  0.2  0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import datasets\n",
    "\n",
    "iris = datasets.load_iris()\n",
    "trainData = pd.concat([pd.DataFrame(iris.data), pd.DataFrame(iris.target)], axis=1)\n",
    "trainData.columns = ['V1', 'V2', 'V3', 'V4', 'y']\n",
    "trainData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "V1 is in processing\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'BinBadRate' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-8-e37e3054377b>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      9\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrainData\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcol\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m   \u001b[1;31m#－1会当成特殊值处理。如果没有－1，则所有取值都参与分箱\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m         \u001b[0mmax_interval\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m5\u001b[0m   \u001b[1;31m#分箱后的最多的箱数\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m         \u001b[0mcutOff\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mChiMerge\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrainData\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcol\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'y'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_interval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmax_interval\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mspecial_attribute\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mminBinPcnt\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     12\u001b[0m         \u001b[0mtrainData\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcol\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;34m'_Bin'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrainData\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcol\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mAssignBin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcutOff\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mspecial_attribute\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m         \u001b[0mmonotone\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBadRateMonotone\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrainData\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcol\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;34m'_Bin'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'y'\u001b[0m\u001b[1;33m)\u001b[0m   \u001b[1;31m# 检验分箱后的单调性是否满足\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-3-3f32e5d01551>\u001b[0m in \u001b[0;36mChiMerge\u001b[1;34m(df, col, target, max_interval, special_attribute, minBinPcnt)\u001b[0m\n\u001b[0;32m     34\u001b[0m             \u001b[0mdf2\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'temp'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdf2\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcol\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m         \u001b[1;31m# 总体bad rate将被用来计算expected bad count\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 36\u001b[1;33m         \u001b[1;33m(\u001b[0m\u001b[0mbinBadRate\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mregroup\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moverallRate\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBinBadRate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'temp'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrantRateIndicator\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     37\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m         \u001b[1;31m# 首先，每个单独的属性值将被分为单独的一组\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'BinBadRate' is not defined"
     ]
    }
   ],
   "source": [
    "##############################################\n",
    "### \n",
    "##############################################\n",
    "num_features = ['V1', 'V2', 'V3', 'V4']\n",
    "# （iii）对连续型变量进行分箱，包括（ii）中的变量\n",
    "continous_merged_dict = {}\n",
    "for col in num_features:\n",
    "    print (\"{} is in processing\".format(col))\n",
    "    if -1 not in set(trainData[col]):   #－1会当成特殊值处理。如果没有－1，则所有取值都参与分箱\n",
    "        max_interval = 5   #分箱后的最多的箱数\n",
    "        cutOff = ChiMerge(trainData, col, 'y', max_interval=max_interval,special_attribute=[],minBinPcnt=0)\n",
    "        trainData[col+'_Bin'] = trainData[col].map(lambda x: AssignBin(x, cutOff,special_attribute=[]))\n",
    "        monotone = BadRateMonotone(trainData, col+'_Bin', 'y')   # 检验分箱后的单调性是否满足\n",
    "        while(not monotone):\n",
    "            # 检验分箱后的单调性是否满足。如果不满足，则缩减分箱的个数。\n",
    "            max_interval -= 1\n",
    "            cutOff = ChiMerge(trainData, col, 'y', max_interval=max_interval, special_attribute=[],\n",
    "                                          minBinPcnt=0)\n",
    "            trainData[col + '_Bin'] = trainData[col].map(lambda x: AssignBin(x, cutOff, special_attribute=[]))\n",
    "            if max_interval == 2:\n",
    "                # 当分箱数为2时，必然单调\n",
    "                break\n",
    "            monotone = BadRateMonotone(trainData, col + '_Bin', 'y')\n",
    "        newVar = col + '_Bin'\n",
    "        trainData[newVar] = trainData[col].map(lambda x: AssignBin(x, cutOff, special_attribute=[]))\n",
    "        var_bin_list.append(newVar)\n",
    "    else:\n",
    "        max_interval = 5\n",
    "        # 如果有－1，则除去－1后，其他取值参与分箱\n",
    "        cutOff = ChiMerge(trainData, col, 'y', max_interval=max_interval, special_attribute=[-1],\n",
    "                                      minBinPcnt=0)\n",
    "        trainData[col + '_Bin'] = trainData[col].map(lambda x: AssignBin(x, cutOff, special_attribute=[-1]))\n",
    "        monotone = BadRateMonotone(trainData, col + '_Bin', 'y',['Bin -1'])\n",
    "        while (not monotone):\n",
    "            max_interval -= 1\n",
    "            # 如果有－1，－1的bad rate不参与单调性检验\n",
    "            cutOff = ChiMerge(trainData, col, 'y', max_interval=max_interval, special_attribute=[-1],\n",
    "                                          minBinPcnt=0)\n",
    "            trainData[col + '_Bin'] = trainData[col].map(lambda x: AssignBin(x, cutOff, special_attribute=[-1]))\n",
    "            if max_interval == 3:\n",
    "                # 当分箱数为3-1=2时，必然单调\n",
    "                break\n",
    "            monotone = BadRateMonotone(trainData, col + '_Bin', 'y',['Bin -1'])\n",
    "        newVar = col + '_Bin'\n",
    "        trainData[newVar] = trainData[col].map(lambda x: AssignBin(x, cutOff, special_attribute=[-1]))\n",
    "        var_bin_list.append(newVar)\n",
    "    continous_merged_dict[col] = cutOff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################\n",
    "### WOE编码\n",
    "##############################################\n",
    "\n",
    "'''\n",
    "第四步：WOE编码、计算IV\n",
    "'''\n",
    "WOE_dict = {}\n",
    "IV_dict = {}\n",
    "# 分箱后的变量进行编码，包括：\n",
    "# 1，初始取值个数小于5，且不需要合并的类别型变量。存放在less_value_features中\n",
    "# 2，初始取值个数小于5，需要合并的类别型变量。合并后新的变量存放在var_bin_list中\n",
    "# 3，初始取值个数超过5，需要合并的类别型变量。合并后新的变量存放在var_bin_list中\n",
    "# 4，连续变量。分箱后新的变量存放在var_bin_list中\n",
    "all_var = var_bin_list  + less_value_features\n",
    "for var in all_var:\n",
    "    woe_iv = CalcWOE(trainData, var, 'y')\n",
    "    WOE_dict[var] = woe_iv['WOE']\n",
    "    IV_dict[var] = woe_iv['IV']\n",
    "\n",
    "#将变量IV值进行降序排列，方便后续挑选变量\n",
    "IV_dict_sorted = sorted(IV_dict.items(), key=lambda x: x[1], reverse=True)\n",
    "\n",
    "IV_values = [i[1] for i in IV_dict_sorted]\n",
    "IV_name = [i[0] for i in IV_dict_sorted]\n",
    "plt.title('feature IV')\n",
    "plt.bar(range(len(IV_values)),IV_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################\n",
    "### 2.5 单变量分析和多变量分析\n",
    "##############################################\n",
    "\n",
    "'''\n",
    "第五步：单变量分析和多变量分析，均基于WOE编码后的值。\n",
    "（1）选择IV高于0.01的变量\n",
    "（2）比较两两线性相关性。如果相关系数的绝对值高于阈值，剔除IV较低的一个\n",
    "'''\n",
    "\n",
    "#选取IV>0.01的变量\n",
    "high_IV = {k:v for k, v in IV_dict.items() if v >= 0.01}\n",
    "high_IV_sorted = sorted(high_IV.items(),key=lambda x:x[1],reverse=True)\n",
    "\n",
    "short_list = high_IV.keys()\n",
    "short_list_2 = []\n",
    "for var in short_list:\n",
    "    newVar = var + '_WOE'\n",
    "    trainData[newVar] = trainData[var].map(WOE_dict[var])\n",
    "    short_list_2.append(newVar)\n",
    "\n",
    "#对于上一步的结果，计算相关系数矩阵，并画出热力图进行数据可视化\n",
    "trainDataWOE = trainData[short_list_2]\n",
    "f, ax = plt.subplots(figsize=(10, 8))\n",
    "corr = trainDataWOE.corr()\n",
    "sns.heatmap(corr, mask=np.zeros_like(corr, dtype=np.bool), cmap=sns.diverging_palette(220, 10, as_cmap=True),square=True, ax=ax)\n",
    "\n",
    "\n",
    "#两两间的线性相关性检验\n",
    "#1，将候选变量按照IV进行降序排列\n",
    "#2，计算第i和第i+1的变量的线性相关系数\n",
    "#3，对于系数超过阈值的两个变量，剔除IV较低的一个\n",
    "deleted_index = []\n",
    "cnt_vars = len(high_IV_sorted)\n",
    "for i in range(cnt_vars):\n",
    "    if i in deleted_index:\n",
    "        continue\n",
    "    x1 = high_IV_sorted[i][0]+\"_WOE\"\n",
    "    for j in range(cnt_vars):\n",
    "        if i == j or j in deleted_index:\n",
    "            continue\n",
    "        y1 = high_IV_sorted[j][0]+\"_WOE\"\n",
    "        roh = np.corrcoef(trainData[x1],trainData[y1])[0,1]\n",
    "        if abs(roh)>0.7:\n",
    "            x1_IV = high_IV_sorted[i][1]\n",
    "            y1_IV = high_IV_sorted[j][1]\n",
    "            if x1_IV > y1_IV:\n",
    "                deleted_index.append(j)\n",
    "            else:\n",
    "                deleted_index.append(i)\n",
    "\n",
    "multi_analysis_vars_1 = [high_IV_sorted[i][0]+\"_WOE\" for i in range(cnt_vars) if i not in deleted_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "多变量分析：VIF\n",
    "'''\n",
    "from statsmodels.stats.outliers_influence import variance_inflation_factor\n",
    "X = np.matrix(trainData[multi_analysis_vars_1])\n",
    "VIF_list = [variance_inflation_factor(X, i) for i in range(X.shape[1])]\n",
    "max_VIF = max(VIF_list)\n",
    "print (max_VIF)\n",
    "# 最大的VIF是1.32267733123，因此这一步认为没有多重共线性\n",
    "multi_analysis = multi_analysis_vars_1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
